import ssl
import urllib.request as ur
import lxml.etree as le
import re

#全局取消证书验证
ssl._create_default_https_context = ssl._create_unverified_context
url = 'https://so.csdn.net/so/search/s.do?q={keyword}&t=blog&platform=pc&p={page}'


def getResponse(url):
    req = ur.Request(
        url=url,
        headers={
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
            'Cookie':'TY_SESSION_ID=7215aeaa-527d-4f01-8c5c-81fa5f73665e; uuid_tt_dd=10_17003094720-1594386142439-971034; dc_session_id=10_1594386142439.554335; UserName=weixin_44246465; UserInfo=442b178f49cc4f4fb87a501224b46e00; UserToken=442b178f49cc4f4fb87a501224b46e00; UserNick=Mier.; AU=9A0; UN=weixin_44246465; BT=1594386226831; p_uid=U000000; Hm_up_6bcd52f51e9b3dce32bec4a3997715ac=%7B%22islogin%22%3A%7B%22value%22%3A%221%22%2C%22scope%22%3A1%7D%2C%22isonline%22%3A%7B%22value%22%3A%221%22%2C%22scope%22%3A1%7D%2C%22isvip%22%3A%7B%22value%22%3A%220%22%2C%22scope%22%3A1%7D%2C%22uid_%22%3A%7B%22value%22%3A%22weixin_44246465%22%2C%22scope%22%3A1%7D%7D; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_17003094720-1594386142439-971034!5744*1*weixin_44246465; __gads=ID=4c481206b4389900:T=1596542512:S=ALNI_MZnUJ2s1Wh5KzgHtkxHK7OHKvwwzQ; announcement=%257B%2522isLogin%2522%253Atrue%252C%2522announcementUrl%2522%253A%2522https%253A%252F%252Flive.csdn.net%252Froom%252Fyzkskaka%252Fats4dBdZ%253Futm_source%253D908346557%2522%252C%2522announcementCount%2522%253A0%257D; dc_sid=931ae7de5db5e2a0cda2c63a5e287e3b; c_first_ref=www.baidu.com; c_segment=10; c_utm_source=vipqytx_hyy; c_first_page=https%3A//blog.csdn.net/fly910905/article/details/83144267; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1598365256,1598365908,1598949183,1598957419; c_page_id=default; c_ref=https%3A//edu.csdn.net/course/detail/30270; is_advert=1; searchHistoryArray=%255B%2522Python%2522%252C%2522Python_%2522%255D; dc_tos=qfzb32; log_Id_pv=17; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1598962143; log_Id_view=35'
        }
    )
    response = ur.urlopen(req).read()
    return response


if __name__ == '__main__':
    keyword = input('搜索关键词')
    page_start=input('初始页')
    page_end=input('终止页')
    for page in range(page_start,page_end):
        # 访问1级页面
        response = getResponse(
            url='https://so.csdn.net/so/search/s.do?q={keyword}&t=blog&platform=pc&p={page}'.format(
                page=page, keyword=keyword)
        )
        # 二级页面，博客的链接
        hrefs = le.HTML(response).xpath('//div[@class="search-list-con"]/dl/dt/div/a[1]/@href') #html文件的网址
        for href in hrefs:
            response_blog = getResponse(
                url = href,
            )
            title = le.HTML(response_blog).xpath('//h1[@class="title-article"]/text()')[0] #获取标题
            title = re.sub(
                r'[/\\:*"<>|?]','',title
            )
            filepath = './%s.html' % title
            with open(filepath,'wb') as f:
                f.write(response_blog)

            print(title)